

// 想要爬取地址: 
// http://mac-torrents.com/
const https = require('https');
const fs = require('fs');

const puppeteer = require('puppeteer');

const readTorrent = require('read-torrent');
const models = require('./models/index');

let tempArr = [];
let errorArr = [];
let browser = null;
let page = null;
let pageNum = 1;
let saving = false;
async function loopfuncF() {
    if (!browser) browser = await puppeteer.launch({
        headless: false,
        timeout: 10000
    });

    if (!page) page = await browser.newPage();
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36');

    await gotoNewPage();

    // 设置爬取延迟
    setTimeout(() => {
        saveTorrent();
    }, 3000)

}

loopfuncF();

async function gotoNewPage() {
    try {
        if (pageNum == 1) {
            await page.goto('http://mac-torrents.com/');
        } else {
            await page.goto(`http://mac-torrents.com/page/${pageNum}/`)
        }
        await page.waitFor('.wf-cell');
    } catch (error) {
        await gotoNewPage();
    }
}

async function saveTorrent() {
    if (saving) return;
    saving = true;
    let download = await page.$$eval('.wf-cell', (els) => {
        let arr = [];
        els.map((el) => {
            let name = el.dataset['name'];
            let date = el.dataset['date'];
            date = new Date(date).getTime();
            let logo = el.querySelector('img.iso-lazy-load');
            let logoUrl = logo.dataset['src'];
            if (logoUrl) {
                logoUrl = logoUrl.replace('http://', 'https://downproxy.weiyun.com/tx_tls_gate=');
            }
            let itemDownload = el.querySelector('a.download');
            if (itemDownload && itemDownload.href) {
                let downloadUrl = itemDownload.href;
                downloadUrl = downloadUrl.replace('http://', 'https://downproxy.weiyun.com/tx_tls_gate=');
                arr.push({
                    name,
                    date,
                    logoUrl,
                    downloadUrl
                });
            }
        });
        return arr;
    });

    tempArr.push(...download);

    fs.writeFile('magnet.json', JSON.stringify(tempArr), {}, (err) => {
        if (err) return console.log(err);
    });

    fs.writeFile('page.txt', pageNum, {}, (err) => {
        if (err) return console.log(err);
    });

    await download.map(async (item) => {
        try {
            let torrent = await readTorrentFile(item);
            let doc = await models.macSoftware.findOne({
                magnet: `magnet:?xt=urn:btih:${torrent.infoHash}`
            });
            if (doc) return console.log('已存在该磁力: ' + torrent.infoHash);
            let fileModel = new models.macSoftware({
                name: item.name,
                magnet: `magnet:?xt=urn:btih:${torrent.infoHash}`,
                logo: item.logoUrl,
                createDate: torrent.createDate,
                hidden: false
            });
            fileModel.save();
        } catch (error) {
            console.log(`出错文件名： ${item.name}`);
            errorArr.push(item);
            fs.writeFile('errorMagnet.json', JSON.stringify(errorArr), {}, (err) => {
                if (err) return console.log(err);
            });
        }
    });

    console.log('成功保存。');
    saving = false;
    pageNum++;
    if (pageNum > 500) return;
    loopfuncF();
}

function readTorrentFile({ name, downloadUrl }) {
    return new Promise((yes, no) => {
        const file = fs.createWriteStream(`${__dirname}/tempTorrent/${name}.torrent`);
        const request = https.get(downloadUrl, (res) => {
            res.pipe(file);
        }).on('error', (e) => {
            no(e);
        });
        file.on('close', () => {
            readTorrent(`${__dirname}/tempTorrent/${name}.torrent`, (err, torrent) => {
                if (err) {
                    return no(err);
                }
                yes({
                    infoHash: torrent.infoHash,
                    createDate: torrent.created
                });
            });
        });

        file.on('error', (e) => {
            no(e);
        });
    });
}